# coding: utf-8
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
import re
import urllib2
import urllib
import datetime
import threading
import logging
from logging.handlers import TimedRotatingFileHandler
global COUNT
global TIMEOUT
global pro_dict
global pro_data_dict
global pro_certify_dict

if __name__ == "__main__":


    def get_page(url, refer_url, description):
        for i in range(1, COUNT):
            try:
                request = urllib2.Request(url)
                if refer_url:
                    request.add_header("Referer", refer_url)
                request.add_header("Host", "www.yfci.gov.cn")
                request.add_header("User-Agent", "Mozilla/5.0 (compatible; "
                                                 "MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)")
                response = urllib2.urlopen(request, timeout=TIMEOUT)
                the_page = response.read()
                the_page = the_page.decode('gbk')
                return the_page
            except Exception, x:
                if i == COUNT - 1:
                    logger.info(description + " " + x)
                else:
                    continue

    def get_data(urlNames):

        global pro_dict
        global pro_data_dict
        global pro_certify_dict
        #商品房信息发布

        url = "http://www.yfci.gov.cn/HousePresell/user_kfs.aspx"
        the_page = get_page(url, None, '商品房信息发布')

        results=[]
        for theUrl in urlNames:
            str_patt = "href=\'([^\']+)\'>"+theUrl
            str_patt = str_patt.decode('utf-8')
            pattern = re.compile(str_patt)
            temps = pattern.findall(the_page)
            if len(temps)>0:
                results.extend(temps)

        #进入新兴县数据地址
        if len(results) > 0:
            pro_url = url
            url_results=[]
            for findUrl in results:
                result = findUrl + '&page=1'
                 #打开新兴的数据列表界面
                url = url.replace('user_kfs.aspx', result)
                the_page = get_page(url, None, '的数据列表界面')
                 #数据列表的分页地址
                str_patt = "href=\"([^\"]+)\">[[\d]"
                str_patt = str_patt.decode('utf-8')
                pattern = re.compile(str_patt)
                temps=[]
                temps = pattern.findall(the_page)
                temps.insert(0, result)
                url_results.extend(temps)

            url_list = []
            for result in url_results:
                url_list.append(pro_url.replace('user_kfs.aspx', result))
            #pro_dice保存的是项目的url地址及项目的referer，名称，及实际地址
            n_loops = range(len(url_list))
            threads = []
            for u in url_list:
                t = threading.Thread(target=get_project_dict, args=(u,))
                threads.append(t)
            for i in n_loops:
                threads[i].start()
            for i in n_loops:
                threads[i].join()
            #项目详情
            if len(pro_dict) > 0:
                #每个项目的数据,主要是获取里面的分页url
                n_loops = range(len(pro_dict))
                threads = []
                for k, v in pro_dict.items():
                    t = threading.Thread(target=get_project_data, args=(k, v))
                    threads.append(t)
                for i in n_loops:
                    threads[i].start()
                for i in n_loops:
                    threads[i].join()

                logger.info("获取预售证/现售证")
                #获取预售证或现售证，也用多线程
                n_loops = range(len(pro_data_dict))
                threads = []
                for k, l in pro_data_dict.items():
                    t = threading.Thread(target=get_certificate, args=(k, l))
                    threads.append(t)
                for i in n_loops:
                    threads[i].start()
                for i in n_loops:
                    threads[i].join()

                logger.info("获取房间/物业数据")
                if len(pro_certify_dict) > 0:
                    threads = []
                    #每个项目作为一个线程,加快速度
                    n_loops = range(len(pro_certify_dict))
                    for k, v in pro_certify_dict.items():
                        t = threading.Thread(target=get_pro_detail_date, args=(v, ))
                        threads.append(t)
                    for i in n_loops:
                        threads[i].start()
                    for i in n_loops:
                        threads[i].join()

    #商品房信息发布每一页的列表数据
    def get_project_dict(pro_url):
        global pro_dict
        str_patt = "href=\'([^\']+)\'\s+style=\'color:#FF0000\'>([^<]+)[^{290}]+290px\">([^<]+)"
        str_patt = str_patt.decode('utf-8')
        pattern = re.compile(str_patt)
        #每一页对应的项目
        the_page = get_page(pro_url, None, '商品房信息发布列表数据')
        results = pattern.findall(the_page)
        refer_url = pro_url.replace('user_kfs.aspx', 'user_kfs_old.aspx')
        county = get_project_county(the_page)
        #每一页的项目数据,包括地址，项目名称，项目地址
        for r in results:
            url = r[0]
            logger.info("获取项目: " + r[1])
            pro_dict[url] = {"referer": refer_url, "url": url, "name": r[1], "address": r[2],"county":county}

    #获取项目所在区县
    def get_project_county(the_page):
        results=the_page.split("商品房预（销）售项目列表")
        adressName = " "
        if len(results)>0:
            results=results[0].split(">")
            adressName = results[len(results)-1]
        return adressName

    #项目列表数据
    def get_project_data(pro_url, v):
        global pro_data_dict
        str_patt = "href=\"(User_Presell_beian[^\"]+)"
        str_patt = str_patt.decode('utf-8')
        #开发商名字
        kfs_patt = ur'kfsmctd[^/]+>([0-9A-Za-z\u4e00-\u9fa5]+)<'
        #总套数
        zts_patt = "=\"ztstd[^>]+\">([0-9.]+)"
        #总面积
        zmj_patt = "=\"zmjtd[^>]+\">([0-9.]+)"
        #住宅套数
        zzts_patt = "=\"zztstd[^>]+\">([0-9.]+)"
        #住宅面积
        zzmj_patt = "=\"zzmjtd[^>]+\">([0-9.]+)"
        #非住宅套数
        fzzts_patt = "=\"fzztstd[^>]+\">([0-9.]+)"
        #非住宅面积
        fzzmj_patt = "=\"fzzmjtd[^>]+\">([0-9.]+)"
        #已售住宅套数
        yszzts_patt = "=\"yszztstd[^>]+\">([0-9.]+)"
        #已售住宅面积
        yszzmj_patt = "=\"yszzmjtd[^>]+\">([0-9.]+)"
        #剩余住宅套数
        kszzts_patt = "=\"kszztstd[^>]+\">([0-9.]+)"
        #剩余住宅面积
        kszzmj_patt = "=\"kszzmjtd[^>]+\">([0-9.]+)"
        #已售非住宅套数
        ysfzzts_patt = "=\"ysfzztstd[^>]+\">([0-9.]+)"
        #住宅均价
        xmjj_patt = "=\"xmjjtd[^>]+\">([0-9.]+)"
        #住宅均价
        fzzjj_patt = "=\"fzzjjtd[^>]+\">([0-9.]+)"
        #项目详细信息页面, pro_dict的key就是url地址，值是一个dict
        pro_url_list = []
        url = "http://www.yfci.gov.cn/HousePresell/" + v["url"]
        the_page = get_page(url, v["referer"], '项目详细信息页面')
        pattern = re.compile(str_patt)
        pro_result = pattern.findall(the_page)
        #点击项目进去的分页
        #pro_data_dict保存的是<项目详细信息>页面的楼盘列表信息的地址
        if len(pro_result) > 0:
            r = v["url"] + '&page=1'
            pro_result.insert(0, r)
        else:
            pro_result.insert(0, v["url"])
        for r in pro_result:
            pro_url_list.append(r)
        pro_data_dict[pro_url] = pro_url_list
        #匹配开发商
        pattern = re.compile(kfs_patt)
        kfs_results = pattern.findall(the_page)
        v['company'] = ''
        if len(kfs_results) > 0:
            v['company'] = kfs_results[0]
        #总套数
        pattern = re.compile(zts_patt)
        zts_results = pattern.findall(the_page)
        v['zts'] = '0'
        if len(zts_results) > 0:
            v['zts'] = zts_results[0]
        #总面积
        pattern = re.compile(zmj_patt)
        zmj_results = pattern.findall(the_page)
        v['zmj'] = '0'
        if len(zmj_results) > 0:
            v['zmj'] = zmj_results[0]
        #住宅套数
        pattern = re.compile(zzts_patt)
        zzts_results = pattern.findall(the_page)
        v['zzts'] = '0'
        if len(zzts_results) > 0:
            v['zzts'] = zzts_results[0]
        #住宅面积
        pattern = re.compile(zzmj_patt)
        zzmj_results = pattern.findall(the_page)
        v['zzmj'] = '0'
        if len(zzmj_results) > 0:
            v['zzmj'] = zzmj_results[0]
        #非住宅套数
        pattern = re.compile(fzzts_patt)
        fzzt_sresults = pattern.findall(the_page)
        v['fzzts'] = '0'
        if len(fzzt_sresults) > 0:
            v['fzzts'] = fzzt_sresults[0]
        #非住宅面积
        pattern = re.compile(fzzmj_patt)
        fzzmj_results = pattern.findall(the_page)
        v['fzzmj'] = '0'
        if len(fzzmj_results) > 0:
            v['fzzmj'] = fzzmj_results[0]
        #已售住宅套数
        pattern = re.compile(yszzts_patt)
        ys_zz_ts_results = pattern.findall(the_page)
        v['yszzts'] = '0'
        if len(ys_zz_ts_results) > 0:
            v['yszzts'] = ys_zz_ts_results[0]
        #已售住宅面积
        pattern = re.compile(yszzmj_patt)
        yszzmjresults = pattern.findall(the_page)
        v['yszzmj'] = '0'
        if len(yszzmjresults) > 0:
            v['yszzmj'] = yszzmjresults[0]
        #剩余住宅套数
        pattern = re.compile(kszzts_patt)
        kszztsresults = pattern.findall(the_page)
        v['kszzts'] = '0'
        if len(kszztsresults) > 0:
            v['kszzts'] = kszztsresults[0]
        #剩余住宅面积
        pattern = re.compile(kszzmj_patt)
        kszzmjresults = pattern.findall(the_page)
        v['kszzmj'] = '0'
        if len(kszzmjresults) > 0:
            v['kszzmj'] = kszzmjresults[0]
        #已售非住宅套数
        pattern = re.compile(ysfzzts_patt)
        ysfzztsresults = pattern.findall(the_page)
        v['ysfzzts'] = '0'
        if len(ysfzztsresults) > 0:
            v['ysfzzts'] = ysfzztsresults[0]
        #住宅均价
        pattern = re.compile(xmjj_patt)
        xmjjresults = pattern.findall(the_page)
        v['xmjj'] = '0'
        if len(xmjjresults) > 0:
            v['xmjj'] = xmjjresults[0]
        #非住宅均价
        pattern = re.compile(fzzjj_patt)
        fzzjjresults = pattern.findall(the_page)
        v['fzzjj'] = '0'
        if len(fzzjjresults) > 0:
            v['fzzjj'] = fzzjjresults[0]

    #获取项目预售证信息
    def get_certificate(pro_url, pro_url_list):
        global pro_dict
        global pro_certify_dict
        project_name = pro_dict[pro_url]["name"]
        #预售证或者证明号
        certi_patt = '<tr bgcolor=\"white\">([\s\S]+?)</tr>'.decode('utf-8')
        #预售证
        pre_patt = 'target=\"_blank\">([^<]+)</a>'
        pre_patt = pre_patt.decode('utf-8')
        #预售或者证明url地址
        certiurl_parr = 'href="([^\"]+)\">'
        certiurl_parr = certiurl_parr.decode('utf-8')
        #现售坐落
        sell_address_patt = '<td align=\"center\">([^<]+)</td>'
        sell_address_patt = sell_address_patt.decode('utf-8')
        #预售坐落
        pre_address_patt = '<td align=\"center\"><a title=\'([^\']+)\''
        pre_address_patt = pre_address_patt.decode('utf-8')
        #证明号
        sell_patt1 = 'target=\"_blank\">([^<]+)</a>'
        sell_patt1 = sell_patt1.decode('utf-8')
        sell_patt2 = '<td height=\"24\" align=\"center\">([^<]+)</td>'
        sell_patt2 = sell_patt2.decode('utf-8')
        sell_patt3 = 'title=\"([^\"]+)"'
        sell_patt3 = sell_patt3.decode('utf-8')
        for l in pro_url_list:
            url = "http://www.yfci.gov.cn/HousePresell/" + l
            refer_url = url
            the_page = None
            for i in range(1, COUNT):
                try:
                    request = urllib2.Request(url)
                    request.add_header("Referer", refer_url)
                    request.add_header("Host", "www.yfci.gov.cn")
                    request.add_header("User-Agent", "Mozilla/5.0 (compatible; "
                                                     "MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)")
                    response = urllib2.urlopen(request, timeout=TIMEOUT)
                    the_page = response.read()
                    the_page = the_page.decode('gbk')
                    if the_page.find('系统繁忙') > -1:
                        logger.info(project_name + ": 系统繁忙")
                        continue
                    break
                except Exception, x:
                    if i == COUNT - 1:
                        logger.info(project_name + " " + " " + x)
                    else:
                        continue
            #预售证或者证明号信息
            pattern = re.compile(certi_patt)
            try:
                cer_results = pattern.findall(the_page)
            except Exception, x:
                logger.info(project_name + ":获取预售证出错 " + x)
            for cer in cer_results:
                certifi_dict = {'certi': 'null', 'address': 'null', 'url': 'null', "project": project_name, "refer_url": refer_url, "project_url": pro_url}
                pattern = re.compile(certiurl_parr)
                pre_results = pattern.findall(cer)
                if len(pre_results) > 0:
                    ceritifi_url = pre_results[0]
                    certifi_dict['url'] = ceritifi_url
                else:
                #过期
                    continue
                #预售证或者证明号
                pattern = re.compile(pre_patt)
                pre_results = pattern.findall(cer)
                if len(pre_results) > 0:
                    #预售
                    logger.info(project_name + ":预售证<< " + pre_results[0] + ">>")
                    certifi_dict['certi'] = pre_results[0]
                    certifi_dict['ispresell'] = '1'
                    #预售坐落
                    pattern = re.compile(pre_address_patt)
                    address_results = pattern.findall(cer)
                    if len(address_results) > 0:
                        certifi_dict['address'] = address_results[0]
                else:
                    certifi_dict['ispresell'] = '0'
                    #现售坐落
                    pattern = re.compile(sell_address_patt)
                    address_results = pattern.findall(cer)
                    if len(address_results) > 0:
                        certifi_dict['address'] = address_results[0]
                        pattern = re.compile(sell_patt1)
                        sell_results = pattern.findall(cer)
                        if len(sell_results) > 0:
                            certifi_dict['certi'] = sell_results[0]
                            logger.info(project_name + ":现售证<< " + sell_results[0] + ">>")
                        else:
                            pattern = re.compile(sell_patt2)
                            sell_results = pattern.findall(cer)
                            if len(sell_results) > 0:
                                certifi_dict['certi'] = sell_results[0]
                                logger.info(project_name + ":现售证<< " + sell_results[0] + ">>")
                            else:
                                pattern = re.compile(sell_patt3)
                                sell_results = pattern.findall(cer)
                                if len(sell_results) > 0:
                                    certifi_dict['certi'] = sell_results[0]
                                    logger.info(project_name + ":现售证<< " + sell_results[0] + ">>")
                pro_certify_dict[ceritifi_url] = certifi_dict

    def get_pro_detail_date(v):
        #预售房屋建筑面积
        pre_area_patt = '<td id=\"PresellArea\" width=\"245\" align=\"center\" bgcolor=\"#FFFFFF\">([0-9.]+)'
        pre_area_patt = pre_area_patt.decode('utf-8')
        #预售套数
        pre_ts_patt = '<td id=\"djrqtd\" align=\"center\" bgcolor=\"#FFFFFF\">([0-9]+)'
        pre_ts_patt = pre_ts_patt.decode('utf-8')
        #土地使用证号
        pre_landinfo_patt = '<td id=\"landinfo\" width=\"245\" align=\"center\" bgcolor=\"#FFFFFF\">([^<]+)<'
        pre_landinfo_patt = pre_landinfo_patt.decode('utf-8')
        #土地用途
        pre_use_patt = '<td id=\"zczjtd\" align=\"center\" bgcolor=\"#FFFFFF\">([^<]+)<'
        pre_use_patt = pre_use_patt.decode('utf-8')
        #预售证有效日期
        pre_datebegin_patt = '<td id=\"FZDatebegin\" align=\"center\" bgcolor=\"#FFFFFF\">([^<]+)<'
        pre_datebegin_patt = pre_datebegin_patt.decode('utf-8')
        #开盘时间
        pre_kpdate_patt = '<td id=\"kpdate\" align=\"center\" bgcolor=\"#FFFFFF\">([^<]+)<'
        pre_kpdate_patt = pre_kpdate_patt.decode('utf-8')
        #幢号
        bulid_patt = '<td width=\"89\" height=\"28\" align=\"center\">([^>]+)</td>'
        bulid_patt = bulid_patt.decode('utf-8')
        #预售或者证明地址
        presell_patt = 'href=\"([^\"]+)\"[\s]+style=\"color'
        presell_patt = presell_patt.decode('utf-8')
        #房间数据
        room_patt = '(<td width=\"120\" height=\"80\" valign=\"middle\" align=\"center\" style=\"border-left:1px '\
                    'solid #0066FF; border-bottom:1px solid #0066FF;\">[\s\S]+?</a></td>)'
        room_patt = room_patt.decode('utf-8')
        #销售状态
        status_patt = '<Font color=[^>]+>(.*?)</FONT>'
        status_patt = status_patt.decode('utf-8')
        #房间号
        roomnum_patt = '<font[^>]+>(.*?)</font>'
        roomnum_patt = roomnum_patt.decode('utf-8')
        #房间url
        room_link_patt = 'a href=\"([^\"]+)\"><Font'
        room_link_patt = room_link_patt.decode('utf-8')
        #套内面积
        room_area_patt = '<td id=\"HouseArea\" bgcolor=\"white\" class=\"tdfont\">([0-9.]+)'
        room_area_patt = room_area_patt.decode('utf-8')
        #建筑面积
        build_area_patt = '<td id=\"SumBuildArea1\" bgcolor=\"white\" class=\"tdfont\">([0-9.]+)'
        build_area_patt = build_area_patt.decode('utf-8')
        #房间里面的栋号
        room_building_patt = '<td id=\"DongNo\" width=\"38%\" bgcolor=\"white\" class=\"tdfont\">([^<]+)'
        room_building_patt = room_building_patt.decode('utf-8')
        #申报单价
        room_price_patt = '<td id=\"sbdj\" bgcolor=\"white\" class=\"tdfont\">([0-9.,]+)'
        room_price_patt = room_price_patt.decode('utf-8')
        #申报金额
        room_amt_patt = '<td id=\"sbzj\" bgcolor=\"white\" class=\"tdfont\">([0-9.,]+)'
        room_amt_patt = room_amt_patt.decode('utf-8')
        # 如果是现售
        room_list = []
        url = "http://www.yfci.gov.cn/HousePresell/" + v['url']
        refer_url = v["refer_url"]
        project_name = v["project"]
        if v['ispresell'] == '0':
            for i in range(1, COUNT):
                try:
                    request = urllib2.Request(url)
                    request.add_header("Referer", refer_url)
                    request.add_header("Host", "www.yfci.gov.cn")
                    request.add_header("User-Agent", "Mozilla/5.0 (compatible; "
                                                     "MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)")
                    response = urllib2.urlopen(request, timeout=TIMEOUT)
                    the_page = response.read()
                    the_page = the_page.decode('gbk')
                    if the_page.find('系统繁忙') > -1:
                        logger.info(project_name + ": 系统繁忙")
                        continue
                    break
                except Exception, x:
                    if i == COUNT - 1:
                        logger.info(project_name + " " + x)
                    else:
                        continue
            pattern = re.compile(room_patt)
            data_results = pattern.findall(the_page)
            for r in data_results:
                pattern = re.compile(status_patt)
                data_results = pattern.findall(r)
                room_dict = {}
                if len(data_results) > 0:
                    room_dict['status'] = data_results[0]
                else:
                    room_dict['status'] = ''
                pattern = re.compile(roomnum_patt)
                data_results = pattern.findall(r)
                if len(data_results) > 0:
                    room_dict['room'] = data_results[0]
                else:
                    room_dict['room'] = '未知房号'
                    # 获取房间的建筑面积和套内面积
                room_dict['roomarea'] = '0'
                room_dict['buildarea'] = '0'
                pattern = re.compile(room_link_patt)
                data_results = pattern.findall(r)
                if len(data_results) > 0:
                    url = "http://www.yfci.gov.cn/HousePresell/" + data_results[0]
                    for i in range(1, COUNT):
                        try:
                            request = urllib2.Request(url)
                            request.add_header("Referer", refer_url)
                            request.add_header("Host", "www.yfci.gov.cn")
                            request.add_header("User-Agent", "Mozilla/5.0 (compatible; "
                                                             "MSIE 9.0; Windows NT 6.1; "
                                                             "WOW64; Trident/5.0)")
                            response = urllib2.urlopen(request, timeout=TIMEOUT)
                            the_page = response.read()
                            the_page = the_page.decode('gbk')
                            if the_page.find('系统繁忙') > -1:
                                logger.info(project_name + ": 系统繁忙")
                                continue
                            break
                        except Exception, x:
                            if i == COUNT - 1:
                                logger.info(project_name + ": " + x)
                            else:
                                continue
                    # 建筑面积
                    pattern = re.compile(build_area_patt)
                    data_results = pattern.findall(the_page)
                    if len(data_results) > 0:
                        room_dict['buildarea'] = data_results[0]
                    #套内面积
                    pattern = re.compile(room_area_patt)
                    data_results = pattern.findall(the_page)
                    if len(data_results) > 0:
                        room_dict['roomarea'] = data_results[0]
                    #房间里面的栋号
                    room_dict['building'] = '空'.decode('utf-8')
                    pattern = re.compile(room_building_patt)
                    data_results = pattern.findall(the_page)
                    if len(data_results) > 0:
                        room_dict['building'] = data_results[0]
                    #申报单价
                    pattern = re.compile(room_price_patt)
                    data_results = pattern.findall(the_page)
                    room_dict['price'] = '0'.decode('utf-8')
                    if len(data_results) > 0:
                        room_dict['price'] = data_results[0]
                    #申报金额
                    pattern = re.compile(room_amt_patt)
                    data_results = pattern.findall(the_page)
                    room_dict['amt'] = '0'.decode('utf-8')
                    if len(data_results) > 0:
                        room_dict['amt'] = data_results[0]
                room_list.append(room_dict)
            logger.info(project_name + ":" + " (" + v['certi'] + ")" + " @现售房间总数：" + str(len(room_list)))

        # 预售项目数据
        else:
            bulid_list = []
            for i in range(1, COUNT):
                try:
                    request = urllib2.Request(url)
                    request.add_header("Referer", refer_url)
                    request.add_header("Host", "www.yfci.gov.cn")
                    request.add_header("User-Agent", "Mozilla/5.0 (compatible; "
                                                     "MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)")
                    response = urllib2.urlopen(request, timeout=TIMEOUT)
                    the_page = response.read()
                    the_page = the_page.decode('gbk')
                    if the_page.find('系统繁忙') > -1:
                        logger.info(project_name + ": 系统繁忙")
                        continue
                    break
                except Exception, x:
                    if i == COUNT - 1:
                        logger.info(project_name + ": " + x)
                    else:
                        continue
            #预售房屋建筑面积
            pattern = re.compile(pre_area_patt)
            data_results = pattern.findall(the_page)
            v['presellarea'] = -1
            if len(data_results) > 0:
                v['presellarea'] = data_results[0]
            #预售套数
            pattern = re.compile(pre_ts_patt)
            data_results = pattern.findall(the_page)
            v['djrqtd'] = -1
            if len(data_results) > 0:
                v['djrqtd'] = data_results[0]
            #土地使用证号
            pattern = re.compile(pre_landinfo_patt)
            data_results = pattern.findall(the_page)
            v['landinfo'] = 'null'.decode('utf-8')
            if len(data_results) > 0:
                v['landinfo'] = data_results[0]
            #土地用途
            pattern = re.compile(pre_use_patt)
            data_results = pattern.findall(the_page)
            v['zczjtd'] = 'null'
            if len(data_results) > 0:
                v['zczjtd'] = data_results[0]
            #预售证有效日期
            pattern = re.compile(pre_datebegin_patt)
            data_results = pattern.findall(the_page)
            v['zczjtd'] = 'null'
            v['datebegin'] = 'null'
            if len(data_results) > 0:
                v['datebegin'] = data_results[0]
            #开盘时间
            pattern = re.compile(pre_kpdate_patt)
            data_results = pattern.findall(the_page)
            v['kpdate'] = 'null'.decode('utf-8')
            if len(data_results) > 0:
                v['kpdate'] = data_results[0]
            #匹配幢号
            pattern = re.compile(bulid_patt)
            data_results = pattern.findall(the_page)
            for dr in data_results:
                bulid_list.append(dr)
            v['build'] = bulid_list
            pattern = re.compile(presell_patt)
            detail_result = pattern.findall(the_page)
            dr_referurl = url
            #第二个进入地址
            for dr in detail_result:
                dr_url = "http://www.yfci.gov.cn/HousePresell/" + dr
                for i in range(1, COUNT):
                    try:
                        request = urllib2.Request(dr_url)
                        request.add_header("Referer", dr_referurl)
                        request.add_header("Host", "www.yfci.gov.cn")
                        request.add_header("User-Agent", "Mozilla/5.0 (compatible; "
                                                         "MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)")
                        response = urllib2.urlopen(request, timeout=TIMEOUT)
                        the_page = response.read()
                        the_page = the_page.decode('gbk')
                        if the_page.find('系统繁忙') > -1:
                            logger.info(project_name + ": 系统繁忙")
                            continue
                        break
                    except Exception, x:
                        if i == COUNT - 1:
                            logger.info(project_name + ": " + x)
                        else:
                            continue
                pattern = re.compile(room_patt)
                data_results = pattern.findall(the_page)
                for r in data_results:
                    pattern = re.compile(status_patt)
                    sell_results = pattern.findall(r)
                    room_dict = {}
                    if len(sell_results) > 0:
                        room_dict['status'] = sell_results[0]
                    else:
                        room_dict['status'] = ''
                    pattern = re.compile(roomnum_patt)
                    room_results = pattern.findall(r)
                    if len(room_results) > 0:
                        room_dict['room'] = room_results[0]
                    else:
                        room_dict['room'] = '未知房号'
                    #获取房间的建筑面积和套内面积
                    room_dict['roomarea'] = '0'.decode('utf-8')
                    room_dict['buildarea'] = '0'.decode('utf-8')
                    pattern = re.compile(room_link_patt)
                    data_results = pattern.findall(r)
                    if len(data_results) > 0:
                        url = "http://www.yfci.gov.cn/HousePresell/" + data_results[0]
                        for i in range(1, COUNT):
                            try:
                                request = urllib2.Request(url)
                                request.add_header("Referer", refer_url)
                                request.add_header("Host", "www.yfci.gov.cn")
                                request.add_header("User-Agent", "Mozilla/5.0 (compatible; MSIE 9.0; "
                                                                 "Windows NT 6.1; WOW64; Trident/5.0)")
                                response = urllib2.urlopen(request, timeout=TIMEOUT)
                                the_page = response.read()
                                the_page = the_page.decode('gbk')
                                if the_page.find('系统繁忙') > -1:
                                    logger.info(project_name + ": 系统繁忙")
                                    continue
                                break
                            except Exception, x:
                                if i == COUNT - 1:
                                    logger.info(project_name + ": " + x)
                                else:
                                    continue
                        #建筑面积
                        pattern = re.compile(build_area_patt)
                        data_results = pattern.findall(the_page)
                        room_dict['buildarea'] = -1
                        if len(data_results) > 0:
                            room_dict['buildarea'] = data_results[0]
                        #套内面积
                        pattern = re.compile(room_area_patt)
                        data_results = pattern.findall(the_page)
                        room_dict['roomarea'] = -1
                        if len(data_results) > 0:
                            room_dict['roomarea'] = data_results[0]
                        #房间里面的栋号
                        room_dict['building'] = '空'.decode('utf-8')
                        pattern = re.compile(room_building_patt)
                        data_results = pattern.findall(the_page)
                        if len(data_results) > 0:
                            room_dict['building'] = data_results[0]
                        #申报单价
                        pattern = re.compile(room_price_patt)
                        data_results = pattern.findall(the_page)
                        room_dict['price'] = '0'.decode('utf-8')
                        if len(data_results) > 0:
                            room_dict['price'] = data_results[0]
                        #申报金额
                        pattern = re.compile(room_amt_patt)
                        data_results = pattern.findall(the_page)
                        room_dict['amt'] = '0'.decode('utf-8')
                        if len(data_results) > 0:
                            room_dict['amt'] = data_results[0]
                    room_list.append(room_dict)
            logger.info(project_name + ":" + "(" + v['certi'] + ")" + " #预售房间总数：" + str(len(room_list)))

        v['room_list'] = room_list

    def post_all_data():
        try:
            print pro_dict
            print "==============="
            print pro_certify_dict
            for k, v in pro_dict.items():
                has_room = 'Flase'
                p_dict = {k:  v}
                certi_list = []
                for sk, sv in pro_certify_dict.items():
                    if sv["project_url"] == k:
                        room_list = sv['room_list']
                        if len(room_list) > 0:
                            has_room = 'True'
                            certi_list.insert(0, sv)
                if has_room == 'True':
                    logger.info(v['name'] + "数据保存成功！")
                else:
                    logger.info(v['name'] + "没有房间数据！")
        except Exception, x:
            logger.info("保存数据出错：" + str(x))

    logger = logging.getLogger("root")
    formatter = logging.Formatter('%(asctime)s level-%(levelname)-8s %(message)s')
    fileTimeHandler = TimedRotatingFileHandler("fdc.log", "H", 1, 10)
    fileTimeHandler.suffix = "%Y%m%d-%H%M.log"
    fileTimeHandler.setFormatter(formatter)
    logging.basicConfig(level=logging.INFO)
    fileTimeHandler.setFormatter(formatter)
    logger.addHandler(fileTimeHandler)
    logger.info("开始于：" + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
    COUNT = 60
    TIMEOUT = 60
    pro_dict = {}
    pro_data_dict = {}
    pro_certify_dict = {}
    #先删除当天的数据
    #delete_data()
    findAdress=[]
    findAdress.append("新兴")
    #findAdress.append("云浮")
    get_data(findAdress)

    post_all_data()
    logger.info("全部完成:" + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
